#!/usr/bin/env node

/**
 * 网页内容抓取工具 - 替代WebFetch
 * 使用curl + jsdom来解析网页内容
 */

const https = require('https');
const http = require('http');
const { URL } = require('url');
const { exec } = require('child_process');
const util = require('util');

const execPromise = util.promisify(exec);

async function fetchWebPage(url, options = {}) {
    const {
        timeout = 30000,
        userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        followRedirects = true
    } = options;

    try {
        // 构建curl命令
        const curlCommand = [
            'curl',
            '-s', // silent mode
            '-L', // follow redirects
            '--max-time', Math.floor(timeout / 1000),
            '--user-agent', `"${userAgent}"`,
            '-H', '"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"',
            '-H', '"Accept-Language: zh-CN,zh;q=0.9,en;q=0.8"',
            '-H', '"Accept-Encoding: gzip, deflate"',
            '--compressed',
            `"${url}"`
        ].join(' ');

        const { stdout, stderr } = await execPromise(curlCommand);

        if (stderr) {
            console.warn('Curl警告:', stderr);
        }

        return {
            success: true,
            url,
            content: stdout,
            headers: {},
            status: 200
        };

    } catch (error) {
        return {
            success: false,
            url,
            error: error.message,
            content: null
        };
    }
}

function extractTextFromHtml(html, maxLength = 10000) {
    // 简单的HTML文本提取
    let text = html
        .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
        .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
        .replace(/<[^>]+>/g, ' ')
        .replace(/\s+/g, ' ')
        .trim();

    if (text.length > maxLength) {
        text = text.substring(0, maxLength) + '...';
    }

    return text;
}

function extractMetadata(html) {
    const metadata = {};

    // 提取title
    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
    if (titleMatch) {
        metadata.title = titleMatch[1].trim();
    }

    // 提取description
    const descMatch = html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["'][^>]*>/i);
    if (descMatch) {
        metadata.description = descMatch[1].trim();
    }

    // 提取keywords
    const keywordsMatch = html.match(/<meta[^>]+name=["']keywords["'][^>]+content=["']([^"']+)["'][^>]*>/i);
    if (keywordsMatch) {
        metadata.keywords = keywordsMatch[1].trim();
    }

    return metadata;
}

// 命令行接口
if (require.main === module) {
    const args = process.argv.slice(2);

    if (args.length === 0) {
        console.log('用法: node web-fetch.js <URL> [options]');
        console.log('示例: node web-fetch.js https://example.com');
        process.exit(1);
    }

    const url = args[0];
    const options = {};

    // 解析简单选项
    if (args.includes('--text-only')) {
        options.textOnly = true;
    }

    if (args.includes('--metadata')) {
        options.metadata = true;
    }

    fetchWebPage(url, options)
        .then(result => {
            if (result.success) {
                console.log('=== 抓取成功 ===');
                console.log('URL:', result.url);
                console.log('内容长度:', result.content.length, '字符');

                if (options.metadata !== false) {
                    const metadata = extractMetadata(result.content);
                    if (Object.keys(metadata).length > 0) {
                        console.log('\n=== 页面元数据 ===');
                        Object.entries(metadata).forEach(([key, value]) => {
                            console.log(`${key}: ${value}`);
                        });
                    }
                }

                if (options.textOnly) {
                    console.log('\n=== 页面文本内容 ===');
                    console.log(extractTextFromHtml(result.content));
                } else {
                    console.log('\n=== 页面HTML内容 ===');
                    console.log(result.content);
                }
            } else {
                console.error('抓取失败:', result.error);
                process.exit(1);
            }
        })
        .catch(error => {
            console.error('执行错误:', error);
            process.exit(1);
        });
}

module.exports = {
    fetchWebPage,
    extractTextFromHtml,
    extractMetadata
};